// SPDX-License-Identifier: GPL-2.0
/* kernel/rwsem.c: R/W semaphores, public implementation
 *
 * Written by David Howells (dhowells@redhat.com).
 * Derived from asm-i386/semaphore.h
 *
 * Writer lock-stealing by Alex Shi <alex.shi@intel.com>
 * and Michel Lespinasse <walken@google.com>
 *
 * Optimistic spinning by Tim Chen <tim.c.chen@intel.com>
 * and Davidlohr Bueso <davidlohr@hp.com>. Based on mutexes.
 *
 * Rwsem count bit fields re-definition and rwsem rearchitecture by
 * Waiman Long <longman@redhat.com> and
 * Peter Zijlstra <peterz@infradead.org>.
 */

#include <linux/types.h>
#include <linux/kernel.h>
#include <linux/sched.h>
#include <linux/sched/rt.h>
#include <linux/sched/task.h>
#include <linux/sched/debug.h>
#include <linux/sched/wake_q.h>
#include <linux/sched/signal.h>
#include <linux/sched/clock.h>
#include <linux/export.h>
#include <linux/rwsem.h>
#include <linux/atomic.h>
#include <linux/hung_task.h>
#include <trace/events/lock.h>

#define RT_MUTEX_BUILD_MUTEX
#include "_private_/rtmutex.c"

#define rwbase_set_and_save_current_state(state) \
    set_current_state(state)

#define rwbase_restore_current_state() \
    __set_current_state(TASK_RUNNING)

#define rwbase_rtmutex_lock_state(rtm, state) \
    __rt_mutex_lock(rtm, state)

#define rwbase_rtmutex_slowlock_locked(rtm, state, wq) \
    __rt_mutex_slowlock_locked(rtm, NULL, state, wq)

#define rwbase_rtmutex_unlock(rtm) \
    __rt_mutex_unlock(rtm)

#define rwbase_rtmutex_trylock(rtm) \
    __rt_mutex_trylock(rtm)

#define rwbase_signal_pending_state(state, current) \
    signal_pending_state(state, current)

#define rwbase_pre_schedule() \
    rt_mutex_pre_schedule()

#define rwbase_schedule() \
    rt_mutex_schedule()

#define rwbase_post_schedule() \
    rt_mutex_post_schedule()

#include "_private_/rwbase_rt.c"

void __init_rwsem(struct rw_semaphore *sem, const char *name,
                  struct lock_class_key *key)
{
    init_rwbase_rt(&(sem)->rwbase);

#ifdef CONFIG_DEBUG_LOCK_ALLOC
    debug_check_no_locks_freed((void *)sem, sizeof(*sem));
    lockdep_init_map_wait(&sem->dep_map, name, key, 0, LD_WAIT_SLEEP);
#endif
}
EXPORT_SYMBOL(__init_rwsem);

static inline void __down_read(struct rw_semaphore *sem)
{
    rwbase_read_lock(&sem->rwbase, TASK_UNINTERRUPTIBLE);
}

static inline int __down_read_interruptible(struct rw_semaphore *sem)
{
    return rwbase_read_lock(&sem->rwbase, TASK_INTERRUPTIBLE);
}

static inline int __down_read_killable(struct rw_semaphore *sem)
{
    return rwbase_read_lock(&sem->rwbase, TASK_KILLABLE);
}

static inline int __down_read_trylock(struct rw_semaphore *sem)
{
    return rwbase_read_trylock(&sem->rwbase);
}

static inline void __up_read(struct rw_semaphore *sem)
{
    rwbase_read_unlock(&sem->rwbase, TASK_NORMAL);
}

static inline void __sched __down_write(struct rw_semaphore *sem)
{
    rwbase_write_lock(&sem->rwbase, TASK_UNINTERRUPTIBLE);
}

static inline int __sched __down_write_killable(struct rw_semaphore *sem)
{
    return rwbase_write_lock(&sem->rwbase, TASK_KILLABLE);
}

static inline int __down_write_trylock(struct rw_semaphore *sem)
{
    return rwbase_write_trylock(&sem->rwbase);
}

static inline void __up_write(struct rw_semaphore *sem)
{
    rwbase_write_unlock(&sem->rwbase);
}

static inline void __downgrade_write(struct rw_semaphore *sem)
{
    rwbase_write_downgrade(&sem->rwbase);
}

/* Debug stubs for the common API */
#define DEBUG_RWSEMS_WARN_ON(c, sem)

static inline void __rwsem_set_reader_owned(struct rw_semaphore *sem,
                                            struct task_struct *owner)
{
}

static inline bool is_rwsem_reader_owned(struct rw_semaphore *sem)
{
    int count = atomic_read(&sem->rwbase.readers);

    return count < 0 && count != READER_BIAS;
}

/*
 * lock for reading
 */
void __sched down_read(struct rw_semaphore *sem)
{
    might_sleep();
    rwsem_acquire_read(&sem->dep_map, 0, 0, _RET_IP_);

    LOCK_CONTENDED(sem, __down_read_trylock, __down_read);
}
EXPORT_SYMBOL(down_read);

int __sched down_read_interruptible(struct rw_semaphore *sem)
{
    might_sleep();
    rwsem_acquire_read(&sem->dep_map, 0, 0, _RET_IP_);

    if (LOCK_CONTENDED_RETURN(sem, __down_read_trylock, __down_read_interruptible))
    {
        rwsem_release(&sem->dep_map, _RET_IP_);
        return -EINTR;
    }

    return 0;
}
EXPORT_SYMBOL(down_read_interruptible);

int __sched down_read_killable(struct rw_semaphore *sem)
{
    might_sleep();
    rwsem_acquire_read(&sem->dep_map, 0, 0, _RET_IP_);

    if (LOCK_CONTENDED_RETURN(sem, __down_read_trylock, __down_read_killable))
    {
        rwsem_release(&sem->dep_map, _RET_IP_);
        return -EINTR;
    }

    return 0;
}
EXPORT_SYMBOL(down_read_killable);

/*
 * trylock for reading -- returns 1 if successful, 0 if contention
 */
int down_read_trylock(struct rw_semaphore *sem)
{
    int ret = __down_read_trylock(sem);

    if (ret == 1)
        rwsem_acquire_read(&sem->dep_map, 0, 1, _RET_IP_);
    return ret;
}
EXPORT_SYMBOL(down_read_trylock);

/*
 * lock for writing
 */
void __sched down_write(struct rw_semaphore *sem)
{
    might_sleep();
    rwsem_acquire(&sem->dep_map, 0, 0, _RET_IP_);
    LOCK_CONTENDED(sem, __down_write_trylock, __down_write);
}
EXPORT_SYMBOL(down_write);

/*
 * lock for writing
 */
int __sched down_write_killable(struct rw_semaphore *sem)
{
    might_sleep();
    rwsem_acquire(&sem->dep_map, 0, 0, _RET_IP_);

    if (LOCK_CONTENDED_RETURN(sem, __down_write_trylock,
                              __down_write_killable))
    {
        rwsem_release(&sem->dep_map, _RET_IP_);
        return -EINTR;
    }

    return 0;
}
EXPORT_SYMBOL(down_write_killable);

/*
 * trylock for writing -- returns 1 if successful, 0 if contention
 */
int down_write_trylock(struct rw_semaphore *sem)
{
    int ret = __down_write_trylock(sem);

    if (ret == 1)
        rwsem_acquire(&sem->dep_map, 0, 1, _RET_IP_);

    return ret;
}
EXPORT_SYMBOL(down_write_trylock);

/*
 * release a read lock
 */
void up_read(struct rw_semaphore *sem)
{
    rwsem_release(&sem->dep_map, _RET_IP_);
    __up_read(sem);
}
EXPORT_SYMBOL(up_read);

/*
 * release a write lock
 */
void up_write(struct rw_semaphore *sem)
{
    rwsem_release(&sem->dep_map, _RET_IP_);
    __up_write(sem);
}
EXPORT_SYMBOL(up_write);

/*
 * downgrade write lock to read lock
 */
void downgrade_write(struct rw_semaphore *sem)
{
    lock_downgrade(&sem->dep_map, _RET_IP_);
    __downgrade_write(sem);
}
EXPORT_SYMBOL(downgrade_write);
